{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import necessary dependencies and settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "pd.options.display.max_colwidth = 200\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sample corpus of text documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The sky is blue and beautiful.</td>\n",
       "      <td>weather</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Love this blue and beautiful sky!</td>\n",
       "      <td>weather</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The quick brown fox jumps over the lazy dog.</td>\n",
       "      <td>animals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A king's breakfast has sausages, ham, bacon, eggs, toast and beans</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I love green eggs, ham, sausages and bacon!</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The brown fox is quick and the blue dog is lazy!</td>\n",
       "      <td>animals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The sky is very blue and the sky is very beautiful today</td>\n",
       "      <td>weather</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>The dog is lazy but the brown fox is quick!</td>\n",
       "      <td>animals</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                             Document Category\n",
       "0                                      The sky is blue and beautiful.  weather\n",
       "1                                   Love this blue and beautiful sky!  weather\n",
       "2                        The quick brown fox jumps over the lazy dog.  animals\n",
       "3  A king's breakfast has sausages, ham, bacon, eggs, toast and beans     food\n",
       "4                         I love green eggs, ham, sausages and bacon!     food\n",
       "5                    The brown fox is quick and the blue dog is lazy!  animals\n",
       "6            The sky is very blue and the sky is very beautiful today  weather\n",
       "7                         The dog is lazy but the brown fox is quick!  animals"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = ['The sky is blue and beautiful.',\n",
    "          'Love this blue and beautiful sky!',\n",
    "          'The quick brown fox jumps over the lazy dog.',\n",
    "          \"A king's breakfast has sausages, ham, bacon, eggs, toast and beans\",\n",
    "          'I love green eggs, ham, sausages and bacon!',\n",
    "          'The brown fox is quick and the blue dog is lazy!',\n",
    "          'The sky is very blue and the sky is very beautiful today',\n",
    "          'The dog is lazy but the brown fox is quick!'    \n",
    "]\n",
    "labels = ['weather', 'weather', 'animals', 'food', 'food', 'animals', 'weather', 'animals']\n",
    "\n",
    "corpus = np.array(corpus)\n",
    "corpus_df = pd.DataFrame({'Document': corpus, \n",
    "                          'Category': labels})\n",
    "corpus_df = corpus_df[['Document', 'Category']]\n",
    "corpus_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple text pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "wpt = nltk.WordPunctTokenizer()\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "def normalize_document(doc):\n",
    "    # lower case and remove special characters\\whitespaces\n",
    "    doc = re.sub(r'[^a-zA-Z\\s]', '', doc, re.I|re.A)\n",
    "    doc = doc.lower()\n",
    "    doc = doc.strip()\n",
    "    # tokenize document\n",
    "    tokens = wpt.tokenize(doc)\n",
    "    # filter stopwords out of document\n",
    "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "    # re-create document from filtered tokens\n",
    "    doc = ' '.join(filtered_tokens)\n",
    "    return doc\n",
    "\n",
    "normalize_corpus = np.vectorize(normalize_document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['sky blue beautiful', 'love blue beautiful sky',\n",
       "       'quick brown fox jumps lazy dog',\n",
       "       'kings breakfast sausages ham bacon eggs toast beans',\n",
       "       'love green eggs ham sausages bacon',\n",
       "       'brown fox quick blue dog lazy', 'sky blue sky beautiful today',\n",
       "       'dog lazy brown fox quick'], dtype='<U51')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_corpus = normalize_corpus(corpus)\n",
    "norm_corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bag of Words Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0],\n",
       "       [1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0],\n",
       "       [1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0],\n",
       "       [0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1],\n",
       "       [0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "cv = CountVectorizer(min_df=0., max_df=1.)\n",
    "cv_matrix = cv.fit_transform(norm_corpus)\n",
    "cv_matrix = cv_matrix.toarray()\n",
    "cv_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bacon</th>\n",
       "      <th>beans</th>\n",
       "      <th>beautiful</th>\n",
       "      <th>blue</th>\n",
       "      <th>breakfast</th>\n",
       "      <th>brown</th>\n",
       "      <th>dog</th>\n",
       "      <th>eggs</th>\n",
       "      <th>fox</th>\n",
       "      <th>green</th>\n",
       "      <th>ham</th>\n",
       "      <th>jumps</th>\n",
       "      <th>kings</th>\n",
       "      <th>lazy</th>\n",
       "      <th>love</th>\n",
       "      <th>quick</th>\n",
       "      <th>sausages</th>\n",
       "      <th>sky</th>\n",
       "      <th>toast</th>\n",
       "      <th>today</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bacon  beans  beautiful  blue  breakfast  brown  dog  eggs  fox  green  \\\n",
       "0      0      0          1     1          0      0    0     0    0      0   \n",
       "1      0      0          1     1          0      0    0     0    0      0   \n",
       "2      0      0          0     0          0      1    1     0    1      0   \n",
       "3      1      1          0     0          1      0    0     1    0      0   \n",
       "4      1      0          0     0          0      0    0     1    0      1   \n",
       "5      0      0          0     1          0      1    1     0    1      0   \n",
       "6      0      0          1     1          0      0    0     0    0      0   \n",
       "7      0      0          0     0          0      1    1     0    1      0   \n",
       "\n",
       "   ham  jumps  kings  lazy  love  quick  sausages  sky  toast  today  \n",
       "0    0      0      0     0     0      0         0    1      0      0  \n",
       "1    0      0      0     0     1      0         0    1      0      0  \n",
       "2    0      1      0     1     0      1         0    0      0      0  \n",
       "3    1      0      1     0     0      0         1    0      1      0  \n",
       "4    1      0      0     0     1      0         1    0      0      0  \n",
       "5    0      0      0     1     0      1         0    0      0      0  \n",
       "6    0      0      0     0     0      0         0    2      0      1  \n",
       "7    0      0      0     1     0      1         0    0      0      0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all unique words in the corpus\n",
    "vocab = cv.get_feature_names()\n",
    "# show document feature vectors\n",
    "pd.DataFrame(cv_matrix, columns=vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bag of N-Grams Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bacon eggs</th>\n",
       "      <th>beautiful sky</th>\n",
       "      <th>beautiful today</th>\n",
       "      <th>blue beautiful</th>\n",
       "      <th>blue dog</th>\n",
       "      <th>blue sky</th>\n",
       "      <th>breakfast sausages</th>\n",
       "      <th>brown fox</th>\n",
       "      <th>dog lazy</th>\n",
       "      <th>eggs ham</th>\n",
       "      <th>...</th>\n",
       "      <th>lazy dog</th>\n",
       "      <th>love blue</th>\n",
       "      <th>love green</th>\n",
       "      <th>quick blue</th>\n",
       "      <th>quick brown</th>\n",
       "      <th>sausages bacon</th>\n",
       "      <th>sausages ham</th>\n",
       "      <th>sky beautiful</th>\n",
       "      <th>sky blue</th>\n",
       "      <th>toast beans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bacon eggs  beautiful sky  beautiful today  blue beautiful  blue dog  \\\n",
       "0           0              0                0               1         0   \n",
       "1           0              1                0               1         0   \n",
       "2           0              0                0               0         0   \n",
       "3           1              0                0               0         0   \n",
       "4           0              0                0               0         0   \n",
       "5           0              0                0               0         1   \n",
       "6           0              0                1               0         0   \n",
       "7           0              0                0               0         0   \n",
       "\n",
       "   blue sky  breakfast sausages  brown fox  dog lazy  eggs ham     ...       \\\n",
       "0         0                   0          0         0         0     ...        \n",
       "1         0                   0          0         0         0     ...        \n",
       "2         0                   0          1         0         0     ...        \n",
       "3         0                   1          0         0         0     ...        \n",
       "4         0                   0          0         0         1     ...        \n",
       "5         0                   0          1         1         0     ...        \n",
       "6         1                   0          0         0         0     ...        \n",
       "7         0                   0          1         1         0     ...        \n",
       "\n",
       "   lazy dog  love blue  love green  quick blue  quick brown  sausages bacon  \\\n",
       "0         0          0           0           0            0               0   \n",
       "1         0          1           0           0            0               0   \n",
       "2         1          0           0           0            1               0   \n",
       "3         0          0           0           0            0               0   \n",
       "4         0          0           1           0            0               1   \n",
       "5         0          0           0           1            0               0   \n",
       "6         0          0           0           0            0               0   \n",
       "7         0          0           0           0            0               0   \n",
       "\n",
       "   sausages ham  sky beautiful  sky blue  toast beans  \n",
       "0             0              0         1            0  \n",
       "1             0              0         0            0  \n",
       "2             0              0         0            0  \n",
       "3             1              0         0            1  \n",
       "4             0              0         0            0  \n",
       "5             0              0         0            0  \n",
       "6             0              1         1            0  \n",
       "7             0              0         0            0  \n",
       "\n",
       "[8 rows x 29 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you can set the n-gram range to 1,2 to get unigrams as well as bigrams\n",
    "bv = CountVectorizer(ngram_range=(2,2))\n",
    "bv_matrix = bv.fit_transform(norm_corpus)\n",
    "\n",
    "bv_matrix = bv_matrix.toarray()\n",
    "vocab = bv.get_feature_names()\n",
    "pd.DataFrame(bv_matrix, columns=vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:1059: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bacon</th>\n",
       "      <th>beans</th>\n",
       "      <th>beautiful</th>\n",
       "      <th>blue</th>\n",
       "      <th>breakfast</th>\n",
       "      <th>brown</th>\n",
       "      <th>dog</th>\n",
       "      <th>eggs</th>\n",
       "      <th>fox</th>\n",
       "      <th>green</th>\n",
       "      <th>ham</th>\n",
       "      <th>jumps</th>\n",
       "      <th>kings</th>\n",
       "      <th>lazy</th>\n",
       "      <th>love</th>\n",
       "      <th>quick</th>\n",
       "      <th>sausages</th>\n",
       "      <th>sky</th>\n",
       "      <th>toast</th>\n",
       "      <th>today</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.32</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.39</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bacon  beans  beautiful  blue  breakfast  brown   dog  eggs   fox  green  \\\n",
       "0   0.00   0.00       0.60  0.53       0.00   0.00  0.00  0.00  0.00   0.00   \n",
       "1   0.00   0.00       0.49  0.43       0.00   0.00  0.00  0.00  0.00   0.00   \n",
       "2   0.00   0.00       0.00  0.00       0.00   0.38  0.38  0.00  0.38   0.00   \n",
       "3   0.32   0.38       0.00  0.00       0.38   0.00  0.00  0.32  0.00   0.00   \n",
       "4   0.39   0.00       0.00  0.00       0.00   0.00  0.00  0.39  0.00   0.47   \n",
       "5   0.00   0.00       0.00  0.37       0.00   0.42  0.42  0.00  0.42   0.00   \n",
       "6   0.00   0.00       0.36  0.32       0.00   0.00  0.00  0.00  0.00   0.00   \n",
       "7   0.00   0.00       0.00  0.00       0.00   0.45  0.45  0.00  0.45   0.00   \n",
       "\n",
       "    ham  jumps  kings  lazy  love  quick  sausages   sky  toast  today  \n",
       "0  0.00   0.00   0.00  0.00  0.00   0.00      0.00  0.60   0.00    0.0  \n",
       "1  0.00   0.00   0.00  0.00  0.57   0.00      0.00  0.49   0.00    0.0  \n",
       "2  0.00   0.53   0.00  0.38  0.00   0.38      0.00  0.00   0.00    0.0  \n",
       "3  0.32   0.00   0.38  0.00  0.00   0.00      0.32  0.00   0.38    0.0  \n",
       "4  0.39   0.00   0.00  0.00  0.39   0.00      0.39  0.00   0.00    0.0  \n",
       "5  0.00   0.00   0.00  0.42  0.00   0.42      0.00  0.00   0.00    0.0  \n",
       "6  0.00   0.00   0.00  0.00  0.00   0.00      0.00  0.72   0.00    0.5  \n",
       "7  0.00   0.00   0.00  0.45  0.00   0.45      0.00  0.00   0.00    0.0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tv = TfidfVectorizer(min_df=0., max_df=1., use_idf=True)\n",
    "tv_matrix = tv.fit_transform(norm_corpus)\n",
    "tv_matrix = tv_matrix.toarray()\n",
    "\n",
    "vocab = tv.get_feature_names()\n",
    "pd.DataFrame(np.round(tv_matrix, 2), columns=vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document Similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.820599</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.192353</td>\n",
       "      <td>0.817246</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.820599</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.225489</td>\n",
       "      <td>0.157845</td>\n",
       "      <td>0.670631</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.791821</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.850516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.506866</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.225489</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.506866</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.192353</td>\n",
       "      <td>0.157845</td>\n",
       "      <td>0.791821</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.115488</td>\n",
       "      <td>0.930989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.817246</td>\n",
       "      <td>0.670631</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.115488</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.850516</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.930989</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  1.000000  0.820599  0.000000  0.000000  0.000000  0.192353  0.817246   \n",
       "1  0.820599  1.000000  0.000000  0.000000  0.225489  0.157845  0.670631   \n",
       "2  0.000000  0.000000  1.000000  0.000000  0.000000  0.791821  0.000000   \n",
       "3  0.000000  0.000000  0.000000  1.000000  0.506866  0.000000  0.000000   \n",
       "4  0.000000  0.225489  0.000000  0.506866  1.000000  0.000000  0.000000   \n",
       "5  0.192353  0.157845  0.791821  0.000000  0.000000  1.000000  0.115488   \n",
       "6  0.817246  0.670631  0.000000  0.000000  0.000000  0.115488  1.000000   \n",
       "7  0.000000  0.000000  0.850516  0.000000  0.000000  0.930989  0.000000   \n",
       "\n",
       "          7  \n",
       "0  0.000000  \n",
       "1  0.000000  \n",
       "2  0.850516  \n",
       "3  0.000000  \n",
       "4  0.000000  \n",
       "5  0.930989  \n",
       "6  0.000000  \n",
       "7  1.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "similarity_matrix = cosine_similarity(tv_matrix)\n",
    "similarity_df = pd.DataFrame(similarity_matrix)\n",
    "similarity_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering documents using similarity features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document\\Cluster 1</th>\n",
       "      <th>Document\\Cluster 2</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Cluster Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>0.253098</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.308539</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.386952</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.489845</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.732945</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>2.69565</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>3.45108</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Document\\Cluster 1 Document\\Cluster 2  Distance Cluster Size\n",
       "0                  2                  7  0.253098            2\n",
       "1                  0                  6  0.308539            2\n",
       "2                  5                  8  0.386952            3\n",
       "3                  1                  9  0.489845            3\n",
       "4                  3                  4  0.732945            2\n",
       "5                 11                 12   2.69565            5\n",
       "6                 10                 13   3.45108            8"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.cluster.hierarchy import dendrogram, linkage\n",
    "\n",
    "Z = linkage(similarity_matrix, 'ward')\n",
    "pd.DataFrame(Z, columns=['Document\\Cluster 1', 'Document\\Cluster 2', \n",
    "                         'Distance', 'Cluster Size'], dtype='object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x1d5d7822438>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfUAAADjCAYAAACcsI0jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHzJJREFUeJzt3X+cVnWd9/HXWyDBMKkgQBTJwkqzRp3VtNzm3rSEteTe\nagVK061GTdssqzVv18xuu6t7+ykqTeWi5dhaaouGd7abo1Jqgo4gKkimgaACJTCAKPi5/zjf0YvL\na2augevMNXPm/Xw8rsdc55zvOedznTkzn+t7vt9zvooIzMzMbODbrd4BmJmZWW04qZuZmRWEk7qZ\nmVlBOKmbmZkVhJO6mZlZQTipm5mZFYSTuvV7kpZIauoHcUySFJKGdrH8PEk/ynMfVax/oaSf7koM\ntSKpQ9L+9Y6jFtLv5I31jsOsJ07qVleSHpN0TNm8UyTN75yOiIMioq3Pg+uliPhaRHwi7/1Imilp\nQUqaqyXdLOldNdz+Ln2x6BQRIyPi0VrF1Sl9cXle0sb0WiZplqTxtd6X2UDjpG6FtTNJSdKQPGKp\nFUmfA74LfA0YC0wELgU+UM+4Su3ql4Eq/UdE7Am8BvifwDhgYT0Sey3PGWX8f9l2mk8e6/dKa/OS\ndpN0rqQ/Slon6VpJr0nLOmuYH5f0Z+C3af7PJT0pab2k2yUdVLLtOZIulzRP0ibgf0gaIelbkh5P\n68yXNKIkpI9I+rOktZL+V8m2drj0Leldkn4v6RlJKySdkub/vaT7JG1I8y+s8jjsBVwEnBkR10fE\npoh4PiJuiogvVijfJGllN8fy8FTj3yDpKUnfTsVuTz+fSVcDjkzl/0nSQ5L+KunXkvYr2W5IOlPS\nI8AjJfPeWHKcL5X0q1S7vlvSG0rWf6+kpel4XybpNkk9XvVIn38JcCKwBjinZJvHS2pPx//3kt5W\ndhw+L2lR2ud/SBpesvwL6SrIKkn/VHYMK50ze0m6StKadN6c35mcJQ1J59NaSX+SdJZKroRIapN0\nsaTfAZuB/SWdmo71RkmPSjqt/Pcq6YuSnk5xTpM0VdlVi79IOq+nY2fF5KRuA82ngWnAu4G9gb+S\n1VRLvRt4C/C+NH0zMBl4HXAvcHVZ+ZnAxcCewHzg34DDgKPIaoJfBF4oKf8u4E3Ae4ALJL2lPMiU\n8G4GLgHGAA1Ae1q8CTgZGAX8PXCGpGlVfPYjgeHADVWUrcb3gO9FxKuANwDXpvl/m36OSpfQ75R0\nAnAe8A9kn+cO4Jqy7U0DjgAO7GJ/04GvAK8GlpMdcySNBn4BfAl4LbCU7NhXLSK2A/8JHJ22eQhw\nBXBa2uYPgLmSdi9Z7R+B44DXA28DTknrHgd8HjiW7LzZoXkoKT9nLgH2AvYnO/9OBk5NZT8JTCE7\nBw4lO07lTgKa0/YeB54GjgdelbbzHUmHlpQfR3YuTAAuAH4IfJTsvD0a+FdJr+/qeFmBRYRfftXt\nBTwGdADPlLw2A/PLyhyT3j8EvKdk2XjgeWAoMAkIYP9u9jcqldkrTc8BripZvhuwBXh7hXU7t79P\nybw/ANPT+wuBn6b3XwJuqPIYfBf4Ttk+hlYo9xHgyR62VRpDE7CywvHuPJa3kyXZ0V18zqEl824G\nPl52nDYD+6XpAP6ubDsBvLHkOP+oZNlU4OH0/mTgzpJlAlYAn+jpM5bNPx14JL2/HPhq2fKlwLtL\njsNHS5Z9E5id3l8BfL1k2QEVPkvpOTMEeA44sGTeaUBbev9b4LSSZceUHl+gDbioh9/rL4HPlPxe\ntwBD0vSeaXtHlJRfCEzL6+/Wr/77ck3d+oNpETGq8wV8qpuy+wE3pEuqz5Al+e1k7cudVnS+SZc+\nv67scv0Gsn/mAKMrlU/zhwN/7CaGJ0vebwZGViizb1fbkHSEpFvTpdr1ZMlodKWyZdYBo1W7NuuP\nkyWshyXdI+n4bsruB3yv5Lj/hSz5Tigps6Limi/p6rjtXbpuRASwQ7NBlSakuDrjPacz3hTzvmlf\nvYqHrOZcrvycGVZW7nFeOjbl26t0nHaYJ2mKpLvSpfRnyL4ElZ4j6yK7OgFZggd4qmT5Fiqfl1Zw\nTuo20KwAppR+CYiI4RHxREmZ0qEHZwInkNWO9iKrhUKWkCqVXws8S3Y5elfj7GobrcBcYN+I2AuY\nXRZPV+4EtlL58m0lm4A9OieUdega0zkdEY9ExAyyZolvAL+Q9Ep2PB6dVpDVNkuP+4iI+H1JmZ0d\n8nE1sE9JnCqdrkZqv34/WbNAZ7wXl8W7R0SUNxl0Fc++JdMTK5QpP2eeJ/siUbpO5zm5w+cr2/bL\ntpeaCK4jawYam77ozqO6c8QGOSd1G2hmAxd3dtKSNCa193ZlT7JEuI4swX2tu41HxAtkl1+/LWnv\nVNM/sqwtthpXA8dI+kdJQyW9VlJDSUx/iYhnJR1O9sWjRxGxnqz99NLUMWoPScNSre6bFVZZBgxX\n1jFvGHA+8OLnkPRRSWPSZ34mzX6BrMPZC2Ttw51mA19S6mSYOoZ9uMpj0ZNfAQenzzQUOJOszbhH\n6di+hax9fxzQ2dnvh8Dp6aqIJL0yHYc9q9jstcApkg6UtAfw5e4KpxrztWTn5Z7p3Pwc0Nlp8lrg\nM5ImSBoF/EsP+38F2e9pDbBN0hTgvVXEbeakbgPO98hqubdI2gjcRdY5qytXkV0KfQJ4MJXvyeeB\nxcA9ZJdzv0Ev/1Yi4s9kl0zPSdtoB96eFn8KuCjFfwEvdVCrZrvfIksY55P9018BnEXW5lpedn3a\n14/IPv8mdrysfRywRFIH2XGdHhFbImIzWSew36VL1++IiBvIjsPPUjPGA2Sdv3ZZRKwFPkzWrr2O\nrKPdArIvY105McW9nux8WAccFhGr0jYXkHVQm0XWmXI5qSNcFfHcTNbP4bdpvd9WsdqnyY7vo2Qd\n51rJvhxC9gXjFmARcB9ZrXsbWbNRpf1vBP6Z7Lz4K9mXvrnVxG6mrPnKzKx/SJfSVwIfiYhb6x1P\nraWa9+yI2K/Hwma95Jq6mdWdpPdJGpWaOc4jaz+u5qpKv6fsuQdTU1PBBLLL+bW6LdFsB07qZtYf\nHEl2t8Basg5v0yJiS/erDBgiu3Xwr2SX3x8ia3YxqzlffjczMysI19TNzMwKwkndzMysIPpiNKWa\nGj16dEyaNKneYZiZmfWZhQsXro2IMT2VG3BJfdKkSSxYsKDeYZiZmfUZSZUeV/wyuV1+lzRc0h8k\n3S9piaSvVCjTpGzYw/b0co9QMzOznZRnTX0r2ahNHekRlfMl3RwR5fee3hER3Q0kYWZmZlXILamn\nkZY60uSw9PL9c2ZmZjnJtfd7GgyjHXga+E1E3F2h2FGSFkm6uXOwiArbaZa0QNKCNWvW5BmymZnZ\ngJVrR7k0elFDGpnoBklvjYgHSorcC0xMl+inkg1KMbnCdlqAFoDGxkbX9su0tEBra72jMLNamTkT\nmpvrHYUNRH1yn3pEPAPcSjYqVOn8DRHRkd7PA4ZJGt0XMRVJayu0t9c7CjOrhfZ2f0m3nZdbTV3S\nGOD5iHhG0gjgWLKhG0vLjAOeiohI40rvRjaEovVSQwO0tdU7CjPbVU1N9Y7ABrI8L7+PB66UNIQs\nWV8bETdJOh0gImYDHwLOkLQN2EI2nrMvr5uZme2EPHu/LwIOqTB/dsn7WcCsvGIwMzMbTPzsdzMz\ns4JwUjczMysIJ3UzM7OCcFI3MzMrCCd1MzOzgnBSNzMzKwgndTMzs4JwUjczMysIJ3UzM7OCcFI3\nMzMrCCd1MzOzgnBSNzMzKwgndTMzs4LILalLGi7pD5Lul7RE0lcqlJGk70taLmmRpEPzisfMzKzo\n8hxPfSvwdxHRIWkYMF/SzRFxV0mZKcDk9DoCuDz9NDMzs17KraYemY40OSy9oqzYCcBVqexdwChJ\n4/OKyczMrMhybVOXNERSO/A08JuIuLusyARgRcn0yjTPzMzMeinXpB4R2yOiAdgHOFzSW3dmO5Ka\nJS2QtGDNmjW1DdLMzKwg+qT3e0Q8A9wKHFe26Alg35LpfdK88vVbIqIxIhrHjBmTX6BmZmYDWJ69\n38dIGpXejwCOBR4uKzYXODn1gn8HsD4iVucVk5mZWZHl2ft9PHClpCFkXx6ujYibJJ0OEBGzgXnA\nVGA5sBk4Ncd4zMzMCi23pB4Ri4BDKsyfXfI+gDPzisHMzGww8RPlzMzMCsJJ3czMrCCc1M3MzArC\nSd3MzKwgnNTNzMwKwkndzMysIJzUzczMCiLPh8+Y2SDX0gKtrfWOYmBpb89+NjXVNYwBZ+ZMaG6u\ndxT155q6meWmtfWlJGXVaWjIXla99nZ/eezkmrqZ5aqhAdra6h2FFZmvarzENXUzM7OCcFI3MzMr\nCCd1MzOzgnBSNzMzK4jckrqkfSXdKulBSUskfaZCmSZJ6yW1p9cFecVjZmZWdHn2ft8GnBMR90ra\nE1go6TcR8WBZuTsi4vgc4zAzMxsUcqupR8TqiLg3vd8IPARMyGt/ZmZmg12ftKlLmgQcAtxdYfFR\nkhZJulnSQV2s3yxpgaQFa9asyTFSMzOzgSv3pC5pJHAdcHZEbChbfC8wMSLeBlwC/LLSNiKiJSIa\nI6JxzJgx+QZsZmY2QOWa1CUNI0voV0fE9eXLI2JDRHSk9/OAYZJG5xmTmZlZUeXZ+13Aj4GHIuLb\nXZQZl8oh6fAUz7q8YjIzMyuyPHu/vxM4CVgsqXNIh/OAiQARMRv4EHCGpG3AFmB6RESOMZmZmRVW\nbkk9IuYD6qHMLGBWXjGYmZkNJn6inJmZWUE4qZuZmRWEk7qZmVlBOKmbmZkVhJO6mZlZQTipm5mZ\nFUTVSV3SfpKOSe9HpJHXzMzMrJ+oKqlL+iTwC+AHadY+dPGcdjMzM6uPamvqZ5I9IW4DQEQ8Arwu\nr6DMzMys96pN6lsj4rnOCUlDAT/O1czMrB+pNqnfJuk8YISkY4GfAzfmF5aZmZn1VrVJ/VxgDbAY\nOA2YB5yfV1BmZmbWe9UO6DICuCIifgggaUiatzmvwMzMzKx3qq2p/zdZEu80Aviv7laQtK+kWyU9\nKGmJpM9UKCNJ35e0XNIiSYdWH7qZmZmVqramPjwiOjonIqJD0h49rLMNOCci7k33tC+U9JuIeLCk\nzBRgcnodAVyefpqZmVkvVVtT31Rai5Z0GLCluxUiYnVE3JvebwQeAiaUFTsBuCoydwGjJI2vOnoz\nMzN7UbU19bOBn0taBQgYB5xY7U4kTQIOAe4uWzQBWFEyvTLNW13tts3MzCxTVVKPiHskvRl4U5q1\nNCKer2ZdSSOB64CzI2LDzgQpqRloBpg4ceLObMLMzKzwqq2pA/wNMCmtc6gkIuKq7laQNIwsoV8d\nEddXKPIEsG/J9D5p3g4iogVoAWhsbPRDb8zMzCqoKqlL+gnwBqAd2J5mB9BlUpck4MfAQxHx7S6K\nzQXOkvQzsg5y6yPCl97NzMx2QrU19UbgwIjoTS35ncBJwGJJ7WneecBEgIiYTfYQm6nAcrJ73k/t\nxfbNzMysRLVJ/QGyznFV16IjYj5Zp7ruygTZYDFmZma2i6pN6qOBByX9AdjaOTMiPpBLVGZmZtZr\n1Sb1C/MMwszMzHZdtbe03ZZ3IGZmZrZrqnqinKR3SLpHUoek5yRtl7RT95ybmZlZPqp9TOwsYAbw\nCNlgLp8ALs0rKDMzM+u9apM6EbEcGBIR2yPi34Hj8gvLzMzMeqvajnKbJb0CaJf0TbJb26r+QmBm\nZmb5qzYxn5TKngVsInu06z/kFZSZmZn1XrVJfVpEPBsRGyLiKxHxOeD4PAMzMzOz3qk2qX+swrxT\nahiHmZmZ7aJu29QlzQBmAq+XNLdk0auAv+QZmJmZmfVOTx3lfk/WKW408K2S+RuBRXkFZWZmZr3X\nbVKPiMeBxyUdA2yJiBckHQC8GVjcFwGamZlZdaptU78dGC5pAnALWW/4OXkFZWZmZr1XbVJXRGwm\nu43tsoj4MHBQtytIV0h6WtIDXSxvkrReUnt6XdC70M3MzKxU1Uld0pHAR4BfpXlDelhnDj0/de6O\niGhIr4uqjMXMzMwqqDapnw18CbghIpZI2h+4tbsVIuJ23EPezMysz/Rm6NXbSqYfBf65Bvs/StIi\n4Ang8xGxpFIhSc1AM8DEiRNrsFszM7Pi6ek+9e9GxNmSbgSifHlEfGAX9n0vMDEiOiRNBX4JTK5U\nMCJagBaAxsbGl8VhZmZmPdfUf5J+/lutdxwRG0rez5N0maTREbG21vsyMzMbDHq6T31h+nmbpDHp\n/Zpa7FjSOOCpiAhJh5O176+rxbbNzMwGox7b1CVdSDY6227ZpLYBl/TUW13SNUATMFrSSuDLwDCA\niJgNfAg4I21vCzA9Inxp3czMbCf11Kb+OeCdwN9ExJ/SvP2ByyV9NiK+09W6ETGju21HxCxgVu9D\nNjMzs0p6uqXtJGBGZ0KHF3u+fxQ4Oc/AzMzMrHd6SurDKnVcS+3qw/IJyczMzHZGT0n9uZ1cZmZm\nZn2sp45yb5e0ocJ8AcNziMfMzMx2Uk+3tPX0fHczMzPrJ6p99ruZmZn1c07qZmZmBeGkbmZmVhBO\n6mZmZgXhpG5mZlYQTupmZmYF4aRuZmZWEE7qZmZmBZFbUpd0haSnJT3QxXJJ+r6k5ZIWSTo0r1jM\nzMwGgzxr6nOA47pZPgWYnF7NwOU5xmJmZlZ4uSX1iLgd+Es3RU4ArorMXcAoSePzisfMzKzo6tmm\nPgFYUTK9Ms0zMzOzndDTKG39gqRmskv0jB07lgsvvJAPfvCDtLW1sW7dOpqbm2lpaeHggw9m5MiR\n3HnnncyYMYObbrqJrVu3MnPmTObMmcNhhx0GwMKFCznllFNobW1l99135/jjj+eaa67hyCOPpKOj\ng8WLF7+4zde+9rU0NTVx3XXX0dTUxKpVq1i2bNmLy8ePH09jYyM33ngj733ve1m2bBmPPfbYi8sn\nTZrEAQccwC233ML73/9+FixYwOrVq19cfsABB7D33nvT1ta2059p7Vro6FjIY48V5zMV8fc0GD/T\nmjUdbNq0mFWrivOZivh7Guif6fHHW9ltt91ZurQ4n6n891R1voyIXc25XW9cmgTcFBFvrbDsB0Bb\nRFyTppcCTRGxurttNjY2xoIFC3KIduBqasp+trXVMwqzl/O5aX1hMJxnkhZGRGNP5ep5+X0ucHLq\nBf8OYH1PCd3MzMy6ltvld0nXAE3AaEkrgS8DwwAiYjYwD5gKLAc2A6fmFYuZmdlgkFtSj4gZPSwP\n4My89m9mZjbY+IlyZmZmBeGkbmZmVhBO6mZmZgXhpG5mZlYQTupmZmYF4aRuZmZWEAPiMbFmZjZw\ntKxaRetTT/XZ/to73ghA033L+2yfM8eOpXnvvftsf9VyUjczs5pqfeop2js6aBg5sk/21/DDvkvm\nAO0dHQBO6mZmNjg0jBxJ2yGH1DuMXDTdd1+9Q+iS29TNzMwKwkndzMysIJzUzczMCsJJ3czMrCCc\n1M3MzAoi16Qu6ThJSyUtl3RuheVNktZLak+vC/KMx8zMrMhyu6VN0hDgUuBYYCVwj6S5EfFgWdE7\nIuL4vOIwMzMbLPKsqR8OLI+IRyPiOeBnwAk57s/MzGxQy/PhMxOAFSXTK4EjKpQ7StIi4Ang8xGx\npLyApGagGWDixIk5hGo2ALW0QGtrvaPoXvt3s59NZ9c3jmrMnAnNzfWOwmyX1PuJcvcCEyOiQ9JU\n4JfA5PJCEdECtAA0NjZG34Zo1k+1tkJ7OzQ01DuSLrU1DIBkDtlxBCd1G/DyTOpPAPuWTO+T5r0o\nIjaUvJ8n6TJJoyNibY5xmRVHQwO0tdU7ioGvqaneEZjVRJ5t6vcAkyW9XtIrgOnA3NICksZJUnp/\neIpnXY4xmZmZFVZuNfWI2CbpLODXwBDgiohYIun0tHw28CHgDEnbgC3A9IjoN5fXWxa20Lq4n7dZ\nAu1PZu2WTXP6/6XOmQfPpPkwX+I0M8tDrm3qETEPmFc2b3bJ+1nArDxj2BWti1tpf7KdhnH9t80S\noOHc/p/MAdqfzNotndTNzPJR745y/V7DuAbaTmmrdxiF0DSnqd4hmJkVmh8Ta2ZmVhCuqZtZ/9PX\n9+B33tLWl73gfV+85cA1dTPrfzrvwe8rDQ19e79/e3v/f3CQDUiuqQ9ifd27v7OjXF+2rbu3/QBW\n5HvwfV+85cQ19UGss3d/X2kY19CndxK0P9k+IG5JNDOrFdfUB7ki9+53b3szG2xcUzczMysIJ3Uz\nM7OCcFI3MzMrCCd1MzOzgnBSNzMzKwgndTMzs4LINalLOk7SUknLJZ1bYbkkfT8tXyTp0DzjMTMz\nK7LckrqkIcClwBTgQGCGpAPLik0BJqdXM3B5XvGYmZkVXZ419cOB5RHxaEQ8B/wMOKGszAnAVZG5\nCxglaXyOMZmZmRVWnkl9ArCiZHplmtfbMmZmZlaFAfGYWEnNZJfnATokLe3T/Z+qvtxdn/PnG+BU\n4M9X5M8Ghf98xf50ff759qumUJ5J/Qlg35LpfdK83pYhIlqAlloHaGZmViR5Xn6/B5gs6fWSXgFM\nB+aWlZkLnJx6wb8DWB8Rq3OMyczMrLByq6lHxDZJZwG/BoYAV0TEEkmnp+WzgXnAVGA5sBk4Na94\nzMzMik4RUe8YzMzMrAb8RDkzM7OCcFI3MzMrCCd1MzOzgnBS74KkNknPSupIrz69Nz5PknaX9GNJ\nj0vaKKld0pR6x1UrJb+zztd2SZfUO65akXSWpAWStkqaU+94ak3SayTdIGlTOkdn1jumWpM0XdJD\n6TP+UdLR9Y6pViT9VNKTkjZIWibpE/WOqdYkTU754af1jqXcgHj4TB2dFRE/qncQORhK9iS/dwN/\nJrsD4VpJB0fEY/UMrBYiYmTne0kjgSeBn9cvoppbBfxv4H3AiDrHkodLgeeAsUAD8CtJ90fEkvqG\nVRuSjgW+AZwI/AEo2qOxvw40R8RmSW8G2iTdFxEL6x1YDV1Kdtt2v+Oa+iAUEZsi4sKIeCwiXoiI\nm4A/AYfVO7YcfBB4Grij3oHUSkRcHxG/BNbVO5Zak/RKst/Zv0ZER0TMB/4TOKm+kdXUV4CLIuKu\n9Pf3RES87KFbA1VEPBARmzsn0+sNdQyppiRNB54B/rvesVTipN69/yNpraTfSWqqdzB5kTQWOAAo\nRE2ozMdIgwbVOxCrygHAtohYVjLvfuCgOsVTU2n0ykZgTBpyeqWkWZIKdcVF0mWSNgMPA6vJnkky\n4El6FXAR8Ll6x9IVJ/Wu/QuwP9kAMy3AjZIK822zk6RhwNXAlRHxcL3jqSVJ+5E1MVxZ71isaiOB\nDWXzNgB71iGWPIwFhgEfAo4ma144BDi/nkHVWkR8iux3djRwPbC1vhHVzFeBH0fEynoH0hUn9S5E\nxN0RsTEitkbElcDvyNqeC0PSbsBPyNovz6pzOHk4CZgfEX+qdyBWtQ7gVWXz9gI21iGWPGxJPy+J\niNURsRb4NgX73wIQEdtT88k+wBn1jmdXSWoAjgG+U+9YuuOOctULCjTokCQBPyarOUyNiOfrHFIe\nTibrtGMDxzJgqKTJEfFImvd2CtI0FBF/lbSS7P/Ji7PrFU8fGUox2tSbgEnAn7N/n4wEhkg6MCIO\nrWNcO3BNvQJJoyS9T9JwSUMlfQT4W+D/1Tu2GroceAvw/ojY0lPhgUbSUWRNJ0Xq9Q5AOieHk42p\nMKTzPK13XLUQEZvILtdeJOmVkt4FfIDsilJR/DvwaUmvk/Rq4LPATXWOqSbSZ5ouaaSkIZLeB8yg\nn3Yq66UWsi8nDek1G/gV2V0o/UYh/hHkYBjZLUNvBraTdfaYVtZ5Z8BKbc2nkbVzPamXxnQ+LSKu\nrltgtfUx4PqIKMpl21LnA18umf4oWY/qC+sSTe19CriC7K6FdcAZRbmdLfkqMJrsqsSzwLXAxXWN\nqHaC7FL7bLJK4+PA2RFRPkLngJN69Hf26kdSB/BsRKypX1Qv5wFdzMzMCsKX383MzArCSd3MzKwg\nnNTNzMwKwkndzMysIJzUzczMCsJJ3czMrCCc1M0KII0Z3y5piaT7JZ2THgPc3TqT+mKsckk/knRg\nD2Wm9VTGzHrmpG5WDFsioiEiDgKOBaaw4wNqKpkE5J7UI+ITEfFgD8WmAU7qZrvISd2sYCLiaaAZ\nOEuZSZLukHRveh2Vin4dODrV8D/bTbkXpTIPS7pa0kOSfiFpj7TsPZLuk7RY0hWSdk/z2yQ1pvcd\nki5OVxPukjQ27ecDwP9NsRThOeFmdeGkblZAEfEo2bPhX0f2uNVj06ATJwLfT8XOBe5INfzvdFOu\n3JuAyyLiLWTDon4qPYt+DnBiRBxM9gjqSiNzvRK4KyLeDtwOfDIifg/MBb6QYvnjLn58s0HLSd2s\n+IYBP5S0mGyAm64uc1dbbkVE/C69/ynwLrJE/6eS8RGuJBsEqdxzvDR4yUKyJgAzqxEP6GJWQJL2\nJxuM6GmytvWnyIYw3Y1sEJFKPltlufIBI3ozgMTz8dKAE9vx/yCzmnJN3axgJI0hGyVrVkqgewGr\nI+IF4CSyy/IAG4E9S1btqly5iZKOTO9nAvOBpcAkSW9M808CbutF2OWxmNlOcFI3K4YRnbe0Af8F\n3EI2HCvAZcDHJN1PNpzwpjR/EbA9dVr7bDflyi0FzpT0EPBq4PKIeBY4Ffh5unz/AtkXi2r9DPhC\n6mjnjnJmO8lDr5pZ1SRNAm6KiLfWORQzq8A1dTMzs4JwTd3MzKwgXFM3MzMrCCd1MzOzgnBSNzMz\nKwgndTMzs4JwUjczMysIJ3UzM7OC+P8o4OY3karuJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d5d772def0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 3))\n",
    "plt.title('Hierarchical Clustering Dendrogram')\n",
    "plt.xlabel('Data point')\n",
    "plt.ylabel('Distance')\n",
    "dendrogram(Z)\n",
    "plt.axhline(y=1.0, c='k', ls='--', lw=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document</th>\n",
       "      <th>Category</th>\n",
       "      <th>ClusterLabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The sky is blue and beautiful.</td>\n",
       "      <td>weather</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Love this blue and beautiful sky!</td>\n",
       "      <td>weather</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The quick brown fox jumps over the lazy dog.</td>\n",
       "      <td>animals</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A king's breakfast has sausages, ham, bacon, eggs, toast and beans</td>\n",
       "      <td>food</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I love green eggs, ham, sausages and bacon!</td>\n",
       "      <td>food</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The brown fox is quick and the blue dog is lazy!</td>\n",
       "      <td>animals</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The sky is very blue and the sky is very beautiful today</td>\n",
       "      <td>weather</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>The dog is lazy but the brown fox is quick!</td>\n",
       "      <td>animals</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                             Document  \\\n",
       "0                                      The sky is blue and beautiful.   \n",
       "1                                   Love this blue and beautiful sky!   \n",
       "2                        The quick brown fox jumps over the lazy dog.   \n",
       "3  A king's breakfast has sausages, ham, bacon, eggs, toast and beans   \n",
       "4                         I love green eggs, ham, sausages and bacon!   \n",
       "5                    The brown fox is quick and the blue dog is lazy!   \n",
       "6            The sky is very blue and the sky is very beautiful today   \n",
       "7                         The dog is lazy but the brown fox is quick!   \n",
       "\n",
       "  Category  ClusterLabel  \n",
       "0  weather             2  \n",
       "1  weather             2  \n",
       "2  animals             1  \n",
       "3     food             3  \n",
       "4     food             3  \n",
       "5  animals             1  \n",
       "6  weather             2  \n",
       "7  animals             1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.cluster.hierarchy import fcluster\n",
    "max_dist = 1.0\n",
    "\n",
    "cluster_labels = fcluster(Z, max_dist, criterion='distance')\n",
    "cluster_labels = pd.DataFrame(cluster_labels, columns=['ClusterLabel'])\n",
    "pd.concat([corpus_df, cluster_labels], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topic Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\sklearn\\decomposition\\online_lda.py:508: DeprecationWarning: The default value for 'learning_method' will be changed from 'online' to 'batch' in the release 0.20. This warning was introduced in 0.18.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>T1</th>\n",
       "      <th>T2</th>\n",
       "      <th>T3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.832191</td>\n",
       "      <td>0.083480</td>\n",
       "      <td>0.084329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.863554</td>\n",
       "      <td>0.069100</td>\n",
       "      <td>0.067346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.047794</td>\n",
       "      <td>0.047776</td>\n",
       "      <td>0.904430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.037243</td>\n",
       "      <td>0.925559</td>\n",
       "      <td>0.037198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.049121</td>\n",
       "      <td>0.903076</td>\n",
       "      <td>0.047802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.054901</td>\n",
       "      <td>0.047778</td>\n",
       "      <td>0.897321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.888287</td>\n",
       "      <td>0.055697</td>\n",
       "      <td>0.056016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.055704</td>\n",
       "      <td>0.055689</td>\n",
       "      <td>0.888607</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         T1        T2        T3\n",
       "0  0.832191  0.083480  0.084329\n",
       "1  0.863554  0.069100  0.067346\n",
       "2  0.047794  0.047776  0.904430\n",
       "3  0.037243  0.925559  0.037198\n",
       "4  0.049121  0.903076  0.047802\n",
       "5  0.054901  0.047778  0.897321\n",
       "6  0.888287  0.055697  0.056016\n",
       "7  0.055704  0.055689  0.888607"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "lda = LatentDirichletAllocation(n_topics=3, max_iter=10000, random_state=0)\n",
    "dt_matrix = lda.fit_transform(cv_matrix)\n",
    "features = pd.DataFrame(dt_matrix, columns=['T1', 'T2', 'T3'])\n",
    "features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show topics and their weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('sky', 4.3324395825632624), ('blue', 3.373753174831771), ('beautiful', 3.3323652405224857), ('today', 1.3325579841038182), ('love', 1.3304224288080069)]\n",
      "\n",
      "[('bacon', 2.332695948479998), ('eggs', 2.332695948479998), ('ham', 2.332695948479998), ('sausages', 2.332695948479998), ('love', 1.335454457601996), ('beans', 1.332773525378464), ('breakfast', 1.332773525378464), ('kings', 1.332773525378464), ('toast', 1.332773525378464), ('green', 1.3325433207547732)]\n",
      "\n",
      "[('brown', 3.3323474595768783), ('dog', 3.3323474595768783), ('fox', 3.3323474595768783), ('lazy', 3.3323474595768783), ('quick', 3.3323474595768783), ('jumps', 1.3324193736202712), ('blue', 1.2919635624485213)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tt_matrix = lda.components_\n",
    "for topic_weights in tt_matrix:\n",
    "    topic = [(token, weight) for token, weight in zip(vocab, topic_weights)]\n",
    "    topic = sorted(topic, key=lambda x: -x[1])\n",
    "    topic = [item for item in topic if item[1] > 0.6]\n",
    "    print(topic)\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering documents using topic model features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document</th>\n",
       "      <th>Category</th>\n",
       "      <th>ClusterLabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The sky is blue and beautiful.</td>\n",
       "      <td>weather</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Love this blue and beautiful sky!</td>\n",
       "      <td>weather</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The quick brown fox jumps over the lazy dog.</td>\n",
       "      <td>animals</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A king's breakfast has sausages, ham, bacon, eggs, toast and beans</td>\n",
       "      <td>food</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I love green eggs, ham, sausages and bacon!</td>\n",
       "      <td>food</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The brown fox is quick and the blue dog is lazy!</td>\n",
       "      <td>animals</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The sky is very blue and the sky is very beautiful today</td>\n",
       "      <td>weather</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>The dog is lazy but the brown fox is quick!</td>\n",
       "      <td>animals</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                             Document  \\\n",
       "0                                      The sky is blue and beautiful.   \n",
       "1                                   Love this blue and beautiful sky!   \n",
       "2                        The quick brown fox jumps over the lazy dog.   \n",
       "3  A king's breakfast has sausages, ham, bacon, eggs, toast and beans   \n",
       "4                         I love green eggs, ham, sausages and bacon!   \n",
       "5                    The brown fox is quick and the blue dog is lazy!   \n",
       "6            The sky is very blue and the sky is very beautiful today   \n",
       "7                         The dog is lazy but the brown fox is quick!   \n",
       "\n",
       "  Category  ClusterLabel  \n",
       "0  weather             2  \n",
       "1  weather             2  \n",
       "2  animals             1  \n",
       "3     food             0  \n",
       "4     food             0  \n",
       "5  animals             1  \n",
       "6  weather             2  \n",
       "7  animals             1  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "\n",
    "km = KMeans(n_clusters=3, random_state=0)\n",
    "km.fit_transform(features)\n",
    "cluster_labels = km.labels_\n",
    "cluster_labels = pd.DataFrame(cluster_labels, columns=['ClusterLabel'])\n",
    "pd.concat([corpus_df, cluster_labels], axis=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
